{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 智能问答系统（主文件）\n",
    "\n",
    "## 作业要求\n",
    "在这里我们来搭建一个轻量级智能问答系统，所需要的模块，包括：\n",
    "- 文本预处理：这部分已经帮大家写好，只需要看看代码就可以了。\n",
    "- 搭建意图识别分类器：这部分也给大家写好了，使用fastText来做的意图识别器\n",
    "- 倒排表：这部分大家需要自己去创建，同时也需要考虑相似的单词（课程视频中讲过）\n",
    "- 排序：基于倒排表返回的结果，我们再根据余弦相似度来计算query跟候选问题之间的相似度，最后返回相似度最高的问题的答案。这里，我们将使用BERT来表示句子的向量。 \n",
    "\n",
    "## 文件说明\n",
    "* QA_main.ipynb：主脚本。\n",
    "* 智能客服对话系统实验手册.pdf：操作手册。\n",
    "* fasttext.ipynb：基于FastText的意图分类脚本。\n",
    "* preprocessor.ipynb：文本预处理脚本。\n",
    "* Retrieve.ipynb：搭建倒排表脚本。\n",
    "* 数据源：[project2-QA(阿里云盘)](https://www.aliyundrive.com/s/QrzQdiDbQFt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "## %env KMP_DUPLICATE_LIB_OK=TRUE  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "import pickle\n",
    "import emoji\n",
    "import re\n",
    "import jieba\n",
    "import torch\n",
    "import fasttext\n",
    "from sys import platform\n",
    "from torch.utils.data import DataLoader\n",
    "from transformers import BertTokenizer\n",
    "#from bert_code.model import BertModelTest\n",
    "#from bert_code.utils import test\n",
    "#from bert_code.data import DataPrecessForSentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "ename": "TypeError",
     "evalue": "__cinit__() takes at least 2 positional arguments (0 given)",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-2-a31dbd41b582>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;31m# 读取已经处理好的数据： 导入在preprocessor.ipynb中生成的data/question_answer_pares.pkl文件，并将其保存在变量QApares中\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      2\u001b[0m \u001b[0;32mwith\u001b[0m \u001b[0mopen\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'data/question_answer_pares.pkl'\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;34m'rb'\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mf\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 3\u001b[0;31m     \u001b[0mQApares\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpickle\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mload\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m",
      "\u001b[0;32m/Applications/anaconda3/envs/mypy37/lib/python3.7/site-packages/pandas/_libs/internals.pyx\u001b[0m in \u001b[0;36mpandas._libs.internals.BlockManager.__cinit__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: __cinit__() takes at least 2 positional arguments (0 given)"
     ]
    }
   ],
   "source": [
    "# 读取已经处理好的数据： 导入在preprocessor.ipynb中生成的data/question_answer_pares.pkl文件，并将其保存在变量QApares中\n",
    "with open('data/question_answer_pares.pkl','rb') as f:\n",
    "    QApares = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 导入在Retrieve.ipynb中生成的data/retrieve/invertedList.pkl倒排表文件，并将其保存在变量invertedList中\n",
    "with open('data/retrieve/invertedList.pkl','rb') as f:\n",
    "    invertedList = pickle.load(f)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 这一格的内容是从preprocessor.ipynb中粘贴而来，包含了数据预处理的几个关键函数，这部分用来处理input query string\n",
    "import emoji\n",
    "import re\n",
    "import jieba\n",
    "def clean(content):\n",
    "    content = emoji.demojize(content)\n",
    "    content = re.sub('<.*>','',content)\n",
    "    return content\n",
    "def question_cut(content):\n",
    "    return list(jieba.cut(content))\n",
    "def strip(wordList):\n",
    "    return [word.strip() for word in wordList if word.strip()!='']\n",
    "with open(\"data/stopWord.json\",\"r\") as f:\n",
    "    stopWords = f.read().split(\"\\n\")\n",
    "def rm_stop_word(wordList):\n",
    "    return [word for word in wordList if word not in stopWords]\n",
    "\n",
    "def text_processing(sentence):\n",
    "    sentence = clean(sentence)\n",
    "    sentence = question_cut(sentence)\n",
    "    sentence = strip(sentence)\n",
    "    sentence = rm_stop_word(sentence)\n",
    "    return sentence\n",
    "\n",
    "# 这一格是从Retrieve中粘贴而来，用于生成与输入数据较相近的一些候选问题的index\n",
    "def get_retrieve_result(sentence):\n",
    "    \"\"\"\n",
    "    基于输入句子，并利用倒排表返回candidate sentence ids\n",
    "    \"\"\"\n",
    "    sentence = text_processing(sentence)\n",
    "    candidate = set()\n",
    "    for word in sentence:\n",
    "        if word in invertedList:\n",
    "            candidate = candidate | set(invertedList[word])\n",
    "    return candidate"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Warning : `load_model` does not return WordVectorModel or SupervisedModel any more, but a `FastText` object which is very similar.\n"
     ]
    }
   ],
   "source": [
    "# 加载训练好的fasttext模型用于意图识别\n",
    "intention = fasttext.load_model('model/fasttext.ftz')\n",
    "\n",
    "def get_intention_result(sentence):\n",
    "    '''\n",
    "        输入句子，返回意图识别结果\n",
    "        入参：\n",
    "            sentence:输入的句子\n",
    "        出参:\n",
    "            fasttext_label:fasttext模型的输出，共有两种结果:__label__0和__label__1。__label__0表示闲聊型，__label__1表示任务型\n",
    "    '''\n",
    "    sentence = text_processing(sentence)\n",
    "    sentence = ' '.join(sentence)\n",
    "    fasttext_label = intention.predict(sentence)[0][0]\n",
    "    return fasttext_label"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "loading vocabulary file /Users/zhuochen/.fastNLP/embedding/bert-base-chinese/vocab.txt\n",
      "Load pre-trained BERT parameters from file /Users/zhuochen/.fastNLP/embedding/bert-base-chinese/pytorch_model.bin.\n"
     ]
    }
   ],
   "source": [
    "#导入与bert embedding相关的包，关于mxnet包下载的注意事项参考实验手册\n",
    "#from bert_embedding import BertEmbedding\n",
    "#import mxnet\n",
    "\n",
    "#ctx = mxnet.gpu()\n",
    "#embedding = BertEmbedding(ctx=ctx)\n",
    "\n",
    "#使用faseNLP\n",
    "#https://fastnlp.readthedocs.io/zh/v0.5.0/tutorials/tutorial_3_embedding.html#part-v-bert-embedding\n",
    "import torch\n",
    "from fastNLP import Vocabulary\n",
    "from fastNLP.embeddings import BertEmbedding\n",
    "#初始化\n",
    "vocab = Vocabulary()\n",
    "embed = BertEmbedding(vocab, model_dir_or_name='cn-base', requires_grad=False)\n",
    "\n",
    "#bert向量计算函数\n",
    "words=torch.LongTensor([[vocab.to_index(word) for word in ['鲁迅']]])\n",
    "n=embed(words).numpy()\n",
    "vec_len = n.shape[2]\n",
    "def comm_vec(c):\n",
    "    vec_com = np.zeros(vec_len)\n",
    "    words = torch.LongTensor([[vocab.to_index(word) for word in c]])\n",
    "    x=embed(words).numpy()\n",
    "    vec_com=np.array(x.mean(axis=0)).mean(axis=0)\n",
    "    return vec_com\n",
    "\n",
    "#余弦相似度函数\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "tqdm.pandas(desc='question')\n",
    "\n",
    "def get_best_answer(sentence,candidate):\n",
    "    \"\"\"\n",
    "    sentence: 用户输入query， 已经处理好的\n",
    "    candidate: 通过倒排表返回的候选问题的下标列表\n",
    "    \n",
    "    返回：最佳回复，string形式\n",
    "    \"\"\"\n",
    "    ##sentence向量\n",
    "    sentence=text_processing(sentence=sentence)\n",
    "    sentence_array=comm_vec(c=sentence)\n",
    "    ##备选candidate向量\n",
    "    question=QApares.question_after_preprocessing[list(candidate)]\n",
    "    question_array=np.vstack(question.progress_apply(comm_vec))\n",
    "    ##计算相似度\n",
    "    m=question_array.shape[0]\n",
    "    cos_dict = {}\n",
    "    for i0 in range(m):\n",
    "        cos_num=cosine_similarity([sentence_array,question_array[i0]])[0][1]\n",
    "        cos_dict[i0]=cos_num\n",
    "    ##相似度最大的作为答案\n",
    "    max_index=max(cos_dict, key=cos_dict.get)\n",
    "    return QApares.answer[max_index]\n",
    "    ## TODO： 你需要完成这部分\n",
    "    ##   计算query跟每一个候选问题之间的余弦相似度，最后排序\n",
    "    ##   每个query跟候选问题首先要转化成向量形式，这里使用BERT embedding （可参考第一次项目作业）。 如果你想尝试，其他的embedding方法，也可以自行尝试如tf-idf\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "def QA(sentence):\n",
    "    '''\n",
    "        实现一个智能客服系统，输入一个句子sentence，返回一个回答\n",
    "    '''\n",
    "    # 若意图识别结果为闲聊型，则默认返回'闲聊机器人'\n",
    "    if get_intention_result(sentence)=='__label__0':\n",
    "        return '闲聊机器人'\n",
    "    # 根据倒排表进行检索获得候选问题集\n",
    "    candidate = get_retrieve_result(sentence)\n",
    "    # 若候选问题集大小为0，默认返回'我不明白你在说什么'\n",
    "    if len(candidate)==0:\n",
    "        return '我不明白你在说什么'\n",
    "    \n",
    "    return get_best_answer(sentence, candidate)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Dumping model to file cache /var/folders/xp/l5cz0dnx3cs9dvqlvqvt4k540000gn/T/jieba.cache\n",
      "Loading model cost 1.579 seconds.\n",
      "Prefix dict has been built successfully.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'闲聊机器人'"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试\n",
    "QA('发什么快递')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "question: 100%|██████████| 6092/6092 [09:10<00:00, 11.07it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'是的哦是经营茶叶的哈'"
      ]
     },
     "execution_count": 206,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试\n",
    "QA('发哪种快递')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 207,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'闲聊机器人'"
      ]
     },
     "execution_count": 207,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试\n",
    "QA('怎么退款')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 215,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "question: 100%|██████████| 1942/1942 [03:01<00:00, 10.68it/s]\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'有的哦'"
      ]
     },
     "execution_count": 215,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试\n",
    "QA('怎么才能退货退款呢')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 203,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'闲聊机器人'"
      ]
     },
     "execution_count": 203,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试\n",
    "QA('这个商品有优惠券吗')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'闲聊机器人'"
      ]
     },
     "execution_count": 204,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# 测试\n",
    "QA('一二三四五六七')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
